{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The MIT License (MIT)\n",
    "\n",
    "# Copyright (c) 2020, NVIDIA CORPORATION.\n",
    "\n",
    "# Permission is hereby granted, free of charge, to any person obtaining a copy of\n",
    "# this software and associated documentation files (the \"Software\"), to deal in\n",
    "# the Software without restriction, including without limitation the rights to\n",
    "# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n",
    "# the Software, and to permit persons to whom the Software is furnished to do so,\n",
    "# subject to the following conditions:\n",
    "\n",
    "# The above copyright notice and this permission notice shall be included in all\n",
    "# copies or substantial portions of the Software.\n",
    "\n",
    "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
    "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n",
    "# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n",
    "# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n",
    "# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n",
    "# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial: Feature Engineering for Recommender Systems\n",
    "\n",
    "# 5. Feature Engineering\n",
    "\n",
    "## 5.2. Differences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import cudf\n",
    "import cupy\n",
    "\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "itemid = [1000001]*10 + [1000002]*5 + [1000001]*5 + [1000002]*5 + [1000001]*1 + [1000002]*1 + [1000001]*2 + [1000002]*2\n",
    "itemid += [1000001]*3 + [1000002]*2 + [1000001]*1 + [1000002]*1 + [1000001]*6 + [1000002]*3 + [1000001]*2 + [1000002]*2\n",
    "userid = np.random.choice(list(range(10000)), len(itemid))\n",
    "action = np.random.choice(list(range(2)), len(itemid), p=[0.2, 0.8])\n",
    "\n",
    "price = [100.00]*10 + [25.00]*5 + [100.00]*5 + [30.00]*5 + [125.00]*1 + [30.00]*1 + [125.00]*2 + [30.00]*2\n",
    "price += [110.00]*3 + [30.00]*2 + [110.00]*1 + [20.00]*1 + [90.00]*6 + [20.00]*3 + [90.00]*2 + [20.00]*2\n",
    "\n",
    "timestamp = [pd.to_datetime('2020-01-01')]*15\n",
    "timestamp += [pd.to_datetime('2020-01-02')]*10\n",
    "timestamp += [pd.to_datetime('2020-01-03')]*2\n",
    "timestamp += [pd.to_datetime('2020-01-04')]*4\n",
    "timestamp += [pd.to_datetime('2020-01-05')]*5\n",
    "timestamp += [pd.to_datetime('2020-01-07')]*2\n",
    "timestamp += [pd.to_datetime('2020-01-08')]*9\n",
    "timestamp += [pd.to_datetime('2020-01-09')]*4\n",
    "\n",
    "data = pd.DataFrame({\n",
    "    'itemid': itemid,\n",
    "    'userid': userid,\n",
    "    'price': price,\n",
    "    'action': action,\n",
    "    'timestamp': timestamp\n",
    "})\n",
    "\n",
    "data = cudf.from_pandas(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Theory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Another category of powerful features is to calculate the differences to previous datapoints based on a timestamp. For example, we can calculate if the price changed of a product and how much the price change was."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>itemid</th>\n",
       "      <th>userid</th>\n",
       "      <th>price</th>\n",
       "      <th>action</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>7270</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>860</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000001</td>\n",
       "      <td>5390</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000001</td>\n",
       "      <td>5191</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000001</td>\n",
       "      <td>5734</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1000001</td>\n",
       "      <td>6265</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1000001</td>\n",
       "      <td>466</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1000001</td>\n",
       "      <td>4426</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1000001</td>\n",
       "      <td>5578</td>\n",
       "      <td>100.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1000001</td>\n",
       "      <td>8322</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-01-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    itemid  userid  price  action  timestamp\n",
       "0  1000001    7270  100.0       1 2020-01-01\n",
       "1  1000001     860  100.0       1 2020-01-01\n",
       "2  1000001    5390  100.0       0 2020-01-01\n",
       "3  1000001    5191  100.0       1 2020-01-01\n",
       "4  1000001    5734  100.0       0 2020-01-01\n",
       "5  1000001    6265  100.0       1 2020-01-01\n",
       "6  1000001     466  100.0       1 2020-01-01\n",
       "7  1000001    4426  100.0       1 2020-01-01\n",
       "8  1000001    5578  100.0       1 2020-01-01\n",
       "9  1000001    8322  100.0       0 2020-01-01"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data['itemid']==1000001].head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tree-based or deep learning based models have difficulties processing these relationships on their own. Providing the models with these features can significantly improve the performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "offset = 1\n",
    "data_shift = data[['itemid', 'timestamp', 'price']].groupby(['itemid', 'timestamp']).mean().reset_index()\n",
    "data_shift.columns = ['itemid', 'timestamp', 'mean']\n",
    "data_shift['mean_' + str(offset)] = data_shift['mean'].shift(1)\n",
    "data_shift.loc[data_shift['itemid']!=data_shift['itemid'].shift(1), 'mean_' + str(offset)] = None\n",
    "data_shift['diff_' + str(offset)] = data_shift['mean'] - data_shift['mean_' + str(offset)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>itemid</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean_1</th>\n",
       "      <th>diff_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>100.0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>125.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-04</td>\n",
       "      <td>125.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-05</td>\n",
       "      <td>110.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>-15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-07</td>\n",
       "      <td>110.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-08</td>\n",
       "      <td>90.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>-20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-09</td>\n",
       "      <td>90.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1000002</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>25.0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1000002</td>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>30.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    itemid  timestamp   mean mean_1 diff_1\n",
       "0  1000001 2020-01-01  100.0   <NA>   <NA>\n",
       "1  1000001 2020-01-02  100.0  100.0    0.0\n",
       "2  1000001 2020-01-03  125.0  100.0   25.0\n",
       "3  1000001 2020-01-04  125.0  125.0    0.0\n",
       "4  1000001 2020-01-05  110.0  125.0  -15.0\n",
       "5  1000001 2020-01-07  110.0  110.0    0.0\n",
       "6  1000001 2020-01-08   90.0  110.0  -20.0\n",
       "7  1000001 2020-01-09   90.0   90.0    0.0\n",
       "8  1000002 2020-01-01   25.0   <NA>   <NA>\n",
       "9  1000002 2020-01-02   30.0   25.0    5.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_shift.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>itemid</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>price_diff_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-03</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-04</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-05</td>\n",
       "      <td>-15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-07</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-08</td>\n",
       "      <td>-20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1000001</td>\n",
       "      <td>2020-01-09</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1000002</td>\n",
       "      <td>2020-01-01</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1000002</td>\n",
       "      <td>2020-01-02</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    itemid  timestamp price_diff_1\n",
       "0  1000001 2020-01-01         <NA>\n",
       "1  1000001 2020-01-02          0.0\n",
       "2  1000001 2020-01-03         25.0\n",
       "3  1000001 2020-01-04          0.0\n",
       "4  1000001 2020-01-05        -15.0\n",
       "5  1000001 2020-01-07          0.0\n",
       "6  1000001 2020-01-08        -20.0\n",
       "7  1000001 2020-01-09          0.0\n",
       "8  1000002 2020-01-01         <NA>\n",
       "9  1000002 2020-01-02          5.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_shift.columns = ['itemid', 'timestamp', 'c1', 'c2', 'price_diff_1']\n",
    "data_shift.drop(['c1', 'c2'], inplace=True).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.merge(data_shift, how='left', on=['itemid', 'timestamp'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>itemid</th>\n",
       "      <th>userid</th>\n",
       "      <th>price</th>\n",
       "      <th>action</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>price_diff_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000001</td>\n",
       "      <td>4658</td>\n",
       "      <td>110.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-01-05</td>\n",
       "      <td>-15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000001</td>\n",
       "      <td>1899</td>\n",
       "      <td>110.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2020-01-05</td>\n",
       "      <td>-15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000002</td>\n",
       "      <td>7734</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-05</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000002</td>\n",
       "      <td>1267</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-05</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000001</td>\n",
       "      <td>1528</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2020-01-07</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    itemid  userid  price  action  timestamp  price_diff_1\n",
       "0  1000001    4658  110.0       0 2020-01-05         -15.0\n",
       "1  1000001    1899  110.0       0 2020-01-05         -15.0\n",
       "2  1000002    7734   30.0       1 2020-01-05           0.0\n",
       "3  1000002    1267   30.0       1 2020-01-05           0.0\n",
       "4  1000001    1528  110.0       1 2020-01-07           0.0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can combine techniques of TimeSeries data and chain them together. For example, we can calculate the # of purchases per item and then compare the previous week with a the week, 2, 3 or 5 weeks ago. We can recognize patterns over time. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Practise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import cudf\n",
    "import numpy as np\n",
    "import cupy\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df_train = cudf.read_parquet('../data/train.parquet')\n",
    "df_valid = cudf.read_parquet('../data/valid.parquet')\n",
    "df_test = cudf.read_parquet('../data/test.parquet')\n",
    "\n",
    "df_train['brand'] = df_train['brand'].fillna('UNKNOWN')\n",
    "df_valid['brand'] = df_valid['brand'].fillna('UNKNOWN')\n",
    "df_test['brand'] = df_test['brand'].fillna('UNKNOWN')\n",
    "\n",
    "df_train['cat_0'] = df_train['cat_0'].fillna('UNKNOWN')\n",
    "df_valid['cat_0'] = df_valid['cat_0'].fillna('UNKNOWN')\n",
    "df_test['cat_0'] = df_test['cat_0'].fillna('UNKNOWN')\n",
    "\n",
    "df_train['cat_1'] = df_train['cat_1'].fillna('UNKNOWN')\n",
    "df_valid['cat_1'] = df_valid['cat_1'].fillna('UNKNOWN')\n",
    "df_test['cat_1'] = df_test['cat_1'].fillna('UNKNOWN')\n",
    "\n",
    "df_train['cat_2'] = df_train['cat_2'].fillna('UNKNOWN')\n",
    "df_valid['cat_2'] = df_valid['cat_2'].fillna('UNKNOWN')\n",
    "df_test['cat_2'] = df_test['cat_2'].fillna('UNKNOWN')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "cuDF does not support date32, right now. We use pandas to transform the timestamp in only date values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/conda/envs/nvtabular/lib/python3.7/site-packages/cudf/core/column/column.py:1396: UserWarning: Date32 values are not yet supported so this will be typecast to a Date64 value\n",
      "  UserWarning,\n"
     ]
    }
   ],
   "source": [
    "df_train['date'] = cudf.from_pandas(pd.to_datetime(df_train['timestamp'].to_pandas()).dt.date)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**ToDo**:\n",
    "<li>Let's get the price difference of the previous price to the current price per item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "############### Solution ###############"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "offset = 1\n",
    "data_shift = df_train[['product_id', 'date', 'price']].groupby(['product_id', 'date']).mean().reset_index()\n",
    "data_shift.columns = ['product_id', 'date', 'mean']\n",
    "data_shift['mean_' + str(offset)] = data_shift['mean'].shift(1)\n",
    "data_shift.loc[data_shift['product_id']!=data_shift['product_id'].shift(1), 'mean_' + str(offset)] = None\n",
    "data_shift['diff_' + str(offset)] = data_shift['mean'] - data_shift['mean_' + str(offset)]\n",
    "data_shift.columns = ['product_id', 'date', 'c1', 'c2', 'price_diff_1']\n",
    "data_shift.drop(['c1', 'c2'], inplace=True)\n",
    "df_train = df_train.merge(data_shift, how='left', on=['product_id', 'date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "############### Solution End ###########"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimisation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's compare a CPU with the GPU version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def difference_feature(df, offset):\n",
    "    data_shift = df[['product_id', 'date', 'price']].groupby(['product_id', 'date']).mean().reset_index()\n",
    "    data_shift.columns = ['product_id', 'date', 'mean']\n",
    "    data_shift['mean_' + str(offset)] = data_shift['mean'].shift(offset)\n",
    "    data_shift.loc[data_shift['product_id']!=data_shift['product_id'].shift(offset), 'mean_' + str(offset)] = None\n",
    "    data_shift['diff_' + str(offset)] = data_shift['mean'] - data_shift['mean_' + str(offset)]\n",
    "    data_shift.columns = ['product_id', 'date', 'c1', 'c2', 'price_diff_' + str(offset)]\n",
    "    data_shift.drop(['c1', 'c2'], axis=1, inplace=True)\n",
    "    df = df.merge(data_shift, how='left', on=['product_id', 'date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_pd = df_train.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.2 s, sys: 4.81 s, total: 15 s\n",
      "Wall time: 15 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "_ = difference_feature(df_train_pd, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 196 ms, sys: 252 ms, total: 448 ms\n",
      "Wall time: 444 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "_ = difference_feature(df_train, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In our experiments, we achieved a speedup of 43.1s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We shutdown the kernel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'ok', 'restart': False}"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import IPython\n",
    "\n",
    "app = IPython.Application.instance()\n",
    "app.kernel.do_shutdown(False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
